import com.guchenbo.spark.sql.SparkUtils
import org.apache.spark.sql.types
import org.apache.spark.sql.types.{StringType, StructField, StructType}

/**
 * @author guchenbo
 * @date 2021/12/28
 */
object SparkCsv {

  def main(args: Array[String]): Unit = {
    val spark = SparkUtils.sparkSession("spark csv")
    val path = "/Users/guchenbo/Downloads/boxFileTemp.csv"

    val schema = StructType(List(StructField("variable_name", StringType), StructField("range", StringType)))
    val df = spark.read
      .option("sep", ",")
      .option("encoding", "UTF-8")
      .option("header", true)
      .option("escape", "\"")

      .csv(path)
    df.show()

    println(df.schema)

    var cols: List[String] = List()
    types.StructType(df.schema).foreach(f => {
      val s = f.name + " " + f.dataType.simpleString
      cols = cols :+ s
    })

    println(cols.mkString(","))


  }
}
